package com.wind.rag;

import dev.langchain4j.data.document.Document;
import dev.langchain4j.data.document.Metadata;
import dev.langchain4j.data.document.splitter.DocumentByParagraphSplitter;
import dev.langchain4j.data.segment.TextSegment;
import dev.langchain4j.model.Tokenizer;
import dev.langchain4j.model.openai.OpenAiTokenizer;

import java.util.List;

/**
 * @author wind
 * @version 1.0
 * @description: TODO
 * @date 2025/3/18 20:23
 */
public class splitter {
    public static void main(String[] args) {
        // Create a tokenizer instance
        Tokenizer tokenizer = new OpenAiTokenizer();

        // Create a DocumentSplitter with a max segment size of 1024 tokens
        DocumentByParagraphSplitter splitter = new DocumentByParagraphSplitter(1024, 0, tokenizer);

        // Example text to be split
        String text = "《红楼梦》自诞生以来便形成“红学”研究，涵盖文学、历史、哲学、社会学等多领域。其版本考据、人物分析、主题阐释至今仍是学术热点。作品被翻译成数十种语言，改编为戏曲、影视、动漫等多种形式，成为中国文化的标志性符号。\n" +
                "\n" +
                "《红楼梦》不仅是一部爱情小说，更是一幅封建末世的社会长卷，一曲青春与理想的挽歌。它让读者在繁华与幻灭中，窥见人性的永恒挣扎与超越。";

        // Create a Document instance
        Document document = Document.from(text, Metadata.from("document", "0"));

        // Split the text into paragraphs
        List<TextSegment> segments = splitter.split(document);

        for (TextSegment segment : segments) {
            System.out.println(segment.text());
        }
    }
}
